import pandas as pd
from multiprocessing import Pool, cpu_count

def read_csv_chunk(args):
    filename, start, end, file_type = args
    sep = '\t' if file_type == 'tsv' else ','
    return pd.read_csv(filename, sep=sep, skiprows=range(1, start+1), nrows=end-start, header=0, encoding='utf-8', low_memory=False)

def get_chunks(filename, n_chunks):
    total_lines = sum(1 for line in open(filename)) - 1 
    chunk_size = total_lines // n_chunks

    chunks = [(i * chunk_size, (i + 1) * chunk_size) for i in range(n_chunks)]
    chunks[-1] = (chunks[-1][0], total_lines)
    return chunks

def parallel_read_csv(filename, file_type='csv', n_chunks=None):
    if n_chunks is None:
        n_chunks = cpu_count()

    print(f"Number of CPUs used: {n_chunks}")

    chunks = get_chunks(filename, n_chunks)

    with Pool(n_chunks) as pool:
        dfs = pool.map(read_csv_chunk, [(filename, start, end, file_type) for start, end in chunks])

    return pd.concat(dfs, ignore_index=True)

### ==================================================================================================================================


